{
 "metadata": {
  "name": "",
  "signature": "sha256:a4dc62943a0d749391085bce814022a8115cdb8a3f8ddf8f13489c9ab5caae2b"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import math\n",
      "import numpy as np\n",
      "import h5py\n",
      "import matplotlib.pyplot as plt\n",
      "import tensorflow as tf\n",
      "from tensorflow.python.framework import ops\n",
      "from tf_utils import load_dataset, random_mini_batches, convert_to_one_hot, predict\n",
      "\n",
      "%matplotlib inline\n",
      "np.random.seed(1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def Liner_fuction():\n",
      "    np.random.seed(1)\n",
      "    X = tf.constant(np.random.randn(3,1),name = 'X')\n",
      "    W = tf.constant(np.random.randn(4,3), name = 'W')\n",
      "    b = tf.constant(np.random.randn(4,1),name = 'b')\n",
      "    \n",
      "    Y = tf.add(tf.matmul(W ,X),b)\n",
      "    sess = tf.Session()\n",
      "    result = sess.run(Y)\n",
      "    \n",
      "    sess.close()\n",
      "    \n",
      "    return result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print (\"result:\" + str(Liner_fuction()))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "result:[[-2.15657382]\n",
        " [ 2.95891446]\n",
        " [-1.08926781]\n",
        " [-0.84538042]]\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def sigmoid(z):\n",
      "    x = tf.placeholder(tf.float32, name='x')\n",
      "    sigmoid = tf.sigmoid(x)\n",
      "    with tf.Session() as sess:\n",
      "        result = sess.run(sigmoid,feed_dict={x:z})\n",
      "        \n",
      "    return result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print (sigmoid(0))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.5\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def cost(logits, label):\n",
      "    logits1 = tf. placeholder(tf.float32, name = 'logits')\n",
      "    label1 = tf.placeholder(tf.float32, name = 'label')\n",
      "    \n",
      "    cost = tf.nn.sigmoid_cross_entropy_with_logits(logits = logits1, labels = label1)\n",
      "    with tf.Session() as sess:\n",
      "        result = sess.run(cost, feed_dict={logits1:logits,label1:label})\n",
      "        \n",
      "    return result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "logits = sigmoid([0.2,0.4,0.6,0.8])\n",
      "label = [0 ,0, 1, 1]\n",
      "print (logits)\n",
      "print (cost(logits, label))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[ 0.54983395  0.59868765  0.64565629  0.68997449]\n",
        "[ 1.00538719  1.03664088  0.42154732  0.40652379]"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def one_hot(labels, C):\n",
      "    depth = tf.constant(C)\n",
      "    one_hot = tf.one_hot(labels ,depth, axis = 0)\n",
      "    with tf.Session() as sess:\n",
      "        result = sess.run(one_hot)\n",
      "    \n",
      "    return result\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "labels = [1, 3, 2, 0, 3, 2]\n",
      "print (one_hot(labels, C=4))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[ 0.  0.  0.  1.  0.  0.]\n",
        " [ 1.  0.  0.  0.  0.  0.]\n",
        " [ 0.  0.  1.  0.  0.  1.]\n",
        " [ 0.  1.  0.  0.  1.  0.]]\n"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "with tf.Session() as sess:\n",
      "    print (sess.run(tf.ones([3,2,4])))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[[ 1.  1.  1.  1.]\n",
        "  [ 1.  1.  1.  1.]]\n",
        "\n",
        " [[ 1.  1.  1.  1.]\n",
        "  [ 1.  1.  1.  1.]]\n",
        "\n",
        " [[ 1.  1.  1.  1.]\n",
        "  [ 1.  1.  1.  1.]]]\n"
       ]
      }
     ],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X_train_orig, Y_train_orig, X_test_orig, Y_test_orig, classes = load_dataset()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print (X_train_orig.size)\n",
      "X_train_flatten = X_train_orig.reshape(X_train_orig.shape[0], -1).T\n",
      "print (X_train_flatten/255)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "13271040\n",
        "[[ 0.89019608  0.93333333  0.89411765 ...,  0.92156863  0.91372549\n",
        "   0.90196078]\n",
        " [ 0.8627451   0.90980392  0.8627451  ...,  0.88627451  0.88627451\n",
        "   0.8627451 ]\n",
        " [ 0.83921569  0.8745098   0.81568627 ...,  0.84705882  0.85098039\n",
        "   0.81960784]\n",
        " ..., \n",
        " [ 0.81568627  0.84313725  0.82745098 ...,  0.78431373  0.8         0.79215686]\n",
        " [ 0.81960784  0.8         0.81176471 ...,  0.75294118  0.78823529\n",
        "   0.78039216]\n",
        " [ 0.81960784  0.75294118  0.79215686 ...,  0.71372549  0.77647059\n",
        "   0.77254902]]"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 47
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "Y_train = one_hot(Y_train_orig, 6)\n",
      "Y_train = Y_train.reshape(Y_train.shape[0], -1)\n",
      "print(Y_train)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[ 0.  1.  0. ...,  0.  0.  0.]\n",
        " [ 0.  0.  0. ...,  0.  0.  0.]\n",
        " [ 0.  0.  1. ...,  1.  0.  0.]\n",
        " [ 0.  0.  0. ...,  0.  0.  0.]\n",
        " [ 0.  0.  0. ...,  0.  1.  0.]\n",
        " [ 1.  0.  0. ...,  0.  0.  1.]]\n"
       ]
      }
     ],
     "prompt_number": 50
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "Y_train = convert_to_one_hot(Y_train_orig, 6)\n",
      "print(Y_train)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[ 0.  1.  0. ...,  0.  0.  0.]\n",
        " [ 0.  0.  0. ...,  0.  0.  0.]\n",
        " [ 0.  0.  1. ...,  1.  0.  0.]\n",
        " [ 0.  0.  0. ...,  0.  0.  0.]\n",
        " [ 0.  0.  0. ...,  0.  1.  0.]\n",
        " [ 1.  0.  0. ...,  0.  0.  1.]]\n"
       ]
      }
     ],
     "prompt_number": 51
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X_train_flaten = X_train_orig.reshape(X_train_orig.shape[0], -1).T\n",
      "print (X_train_flaten.shape)\n",
      "X_test_flaten = X_test_orig.reshape(X_test_orig.shape[0], -1).T\n",
      "\n",
      "X_train = X_train_flaten/255\n",
      "X_test = X_test_flaten/255\n",
      "\n",
      "Y_test = convert_to_one_hot(Y_test_orig, 6)\n",
      "Y_train = convert_to_one_hot(Y_train_orig, 6)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(12288, 1080)\n"
       ]
      }
     ],
     "prompt_number": 85
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def creat_placeholder(n_x, n_y):\n",
      "    X = tf.placeholder(tf.float32, shape=(n_x,None) ,name='X')\n",
      "    Y = tf.placeholder(tf.float32, shape=(n_y,None) ,name='Y')\n",
      "    return X ,Y"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 62
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X, Y = creat_placeholder(12288, 6)\n",
      "print (X)\n",
      "print (Y)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Tensor(\"X_2:0\", shape=(12288, ?), dtype=float32)\n",
        "Tensor(\"Y_1:0\", shape=(6, ?), dtype=float32)\n"
       ]
      }
     ],
     "prompt_number": 63
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def initializer_parameters():\n",
      "    tf.set_random_seed(1)\n",
      "    W1 = tf.get_variable('W1', [25,12288], initializer = tf.contrib.layers.xavier_initializer(seed=1))\n",
      "    b1 = tf.get_variable('b1', [25,1], initializer = tf.contrib.layers.xavier_initializer(seed=1))\n",
      "    W2 = tf.get_variable('W2', [12,25], initializer = tf.contrib.layers.xavier_initializer(seed=1))\n",
      "    b2 = tf.get_variable('b2', [12,1], initializer = tf.contrib.layers.xavier_initializer(seed=1))\n",
      "    W3 = tf.get_variable('W3', [6,12], initializer = tf.contrib.layers.xavier_initializer(seed=1))\n",
      "    b3 = tf.get_variable('b3', [6,1], initializer = tf.contrib.layers.xavier_initializer(seed=1))\n",
      "    parameters = {'W1':W1, 'W2':W2, 'W3':W3, 'b1':b1, 'b2':b2, 'b3':b3}\n",
      "\n",
      "    return parameters\n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 75
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "parameters = initializer_parameters()\n",
      "print (parameters['W1'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "ValueError",
       "evalue": "Variable W1 already exists, disallowed. Did you mean to set reuse=True or reuse=tf.AUTO_REUSE in VarScope? Originally defined at:\n\n  File \"<ipython-input-64-690e4f4209e3>\", line 3, in initializer_parameters\n    W1 = tf.get_variable('W1', [25,12288], initializer = tf.contrib.layers.xavier_initializer(seed=1))\n  File \"<ipython-input-74-d4eeeb9e7159>\", line 5, in <module>\n    parameters = initializer_parameters()\n  File \"/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\", line 2883, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-76-7288a9fb74ce>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mparameters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minitializer_parameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'W1'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m<ipython-input-75-07162e20b66a>\u001b[0m in \u001b[0;36minitializer_parameters\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minitializer_parameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_random_seed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mW1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'W1'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m12288\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minitializer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxavier_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mb1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'b1'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minitializer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxavier_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mW2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'W2'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m12\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minitializer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxavier_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint)\u001b[0m\n\u001b[1;32m   1201\u001b[0m       \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1202\u001b[0m       \u001b[0muse_resource\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_resource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcustom_getter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcustom_getter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1203\u001b[0;31m       constraint=constraint)\n\u001b[0m\u001b[1;32m   1204\u001b[0m get_variable_or_local_docstring = (\n\u001b[1;32m   1205\u001b[0m     \"\"\"%s\n",
        "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint)\u001b[0m\n\u001b[1;32m   1090\u001b[0m           \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1091\u001b[0m           \u001b[0muse_resource\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_resource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcustom_getter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcustom_getter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1092\u001b[0;31m           constraint=constraint)\n\u001b[0m\u001b[1;32m   1093\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1094\u001b[0m   def _get_partitioned_variable(self,\n",
        "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, custom_getter, constraint)\u001b[0m\n\u001b[1;32m    423\u001b[0m           \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    424\u001b[0m           \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_resource\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_resource\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 425\u001b[0;31m           constraint=constraint)\n\u001b[0m\u001b[1;32m    426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    427\u001b[0m   def _get_partitioned_variable(\n",
        "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, use_resource, constraint)\u001b[0m\n\u001b[1;32m    392\u001b[0m           \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    393\u001b[0m           \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 394\u001b[0;31m           use_resource=use_resource, constraint=constraint)\n\u001b[0m\u001b[1;32m    395\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    396\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcustom_getter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/variable_scope.py\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape, use_resource, constraint)\u001b[0m\n\u001b[1;32m    740\u001b[0m                          \u001b[0;34m\"reuse=tf.AUTO_REUSE in VarScope? \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    741\u001b[0m                          \"Originally defined at:\\n\\n%s\" % (\n\u001b[0;32m--> 742\u001b[0;31m                              name, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m    743\u001b[0m       \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    744\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mValueError\u001b[0m: Variable W1 already exists, disallowed. Did you mean to set reuse=True or reuse=tf.AUTO_REUSE in VarScope? Originally defined at:\n\n  File \"<ipython-input-64-690e4f4209e3>\", line 3, in initializer_parameters\n    W1 = tf.get_variable('W1', [25,12288], initializer = tf.contrib.layers.xavier_initializer(seed=1))\n  File \"<ipython-input-74-d4eeeb9e7159>\", line 5, in <module>\n    parameters = initializer_parameters()\n  File \"/usr/lib/python3/dist-packages/IPython/core/interactiveshell.py\", line 2883, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n"
       ]
      }
     ],
     "prompt_number": 76
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def forward_propagation(x, parameters):\n",
      "    W1 = parameters['W1']\n",
      "    W2 = parameters['W2']\n",
      "    W3 = parameters['W3']\n",
      "    b1 = parameters['b1']\n",
      "    b2 = parameters['b2']\n",
      "    b3 = parameters['b3']\n",
      "    \n",
      "    out = tf.matmul(W1,x) + b1\n",
      "    out = tf.nn.relu(out)\n",
      "    out = tf.matmul(W2,out) +  b2\n",
      "    out = tf.nn.relu(out)\n",
      "    out = tf.matmul(W3,out) + b3\n",
      "    \n",
      "    return out"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 77
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tf.reset_default_graph()\n",
      "\n",
      "with tf.Session() as sess:\n",
      "    X, Y = creat_placeholder(12288, 6)\n",
      "    parameters = initializer_parameters()\n",
      "    out = forward_propagation(X, parameters)\n",
      "\n",
      "print (out)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Tensor(\"add_2:0\", shape=(6, ?), dtype=float32)\n"
       ]
      }
     ],
     "prompt_number": 78
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def compute_cost(out, y):\n",
      "    logits = tf.transpose(out)\n",
      "    labels = tf.transpose(y)\n",
      "    \n",
      "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels=labels))\n",
      "    return cost"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 81
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tf.reset_default_graph()\n",
      "with tf.Session() as sess:\n",
      "    X, Y = creat_placeholder(12288, 6)\n",
      "    parameters = initializer_parameters()\n",
      "    out = forward_propagation(X, parameters)\n",
      "    cost = compute_cost(out, Y)\n",
      "print (cost)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Tensor(\"Mean:0\", shape=(), dtype=float32)\n"
       ]
      }
     ],
     "prompt_number": 82
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def model(X_train, Y_train, X_test, Y_test, learning_rate = 0.0001,\n",
      "          nums_epoch = 1500, minibatch_size = 32, print_cost = True):\n",
      "    tf.reset_default_graph()\n",
      "    tf.set_random_seed(1)\n",
      "    seed = 3\n",
      "    (n_x, m)= X_train.shape\n",
      "    n_y = Y_train.shape[0]\n",
      "    costs = []\n",
      "    \n",
      "    X, Y = creat_placeholder(n_x, n_y)\n",
      "    parameters = initializer_parameters()\n",
      "    out = forward_propagation(X, parameters)\n",
      "    cost = compute_cost(out, Y)\n",
      "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
      "    \n",
      "    init = tf.global_variables_initializer()\n",
      "    \n",
      "    with tf.Session() as sess:\n",
      "        sess.run(init)\n",
      "        for epoch in range(nums_epoch):\n",
      "            epoch_cost = 0\n",
      "            num_minibatches = int(m / minibatch_size)\n",
      "            seed = seed+1\n",
      "            minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed)\n",
      "            \n",
      "            for minibatch in minibatches:\n",
      "                (minibatch_X, minibatch_Y) = minibatch\n",
      "                _, minibatch_cost = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y:minibatch_Y})\n",
      "                epoch_cost += minibatch_cost / num_minibatches\n",
      "            \n",
      "            if print_cost == True and epoch % 100 ==0:\n",
      "                print (\"Cost after epoch %i: %f\" % (epoch, epoch_cost))\n",
      "            if print_cost == True and epoch%5 == 0:\n",
      "                costs.append(epoch_cost)\n",
      "        \n",
      "        plt.plot(np.squeeze(costs))\n",
      "        plt.ylabel('cost')\n",
      "        plt.xlabel('iterations (per tens)')\n",
      "        plt.title(\"Learning rate = \" + str(learning_rate))\n",
      "        plt.show()\n",
      "        \n",
      "        parameters = sess.run(parameters)\n",
      "        print(\"Parameters have benn trained!\")\n",
      "        correct_prediction = tf.equal(tf.argmax(out), tf.argmax(Y))\n",
      "        \n",
      "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
      "        \n",
      "        print(\"Train Accuracy:\", accuracy.eval({X:X_train, Y:Y_train}))\n",
      "        print (\"Test Accuracy:\", accuracy.eval({X:X_test, Y:Y_test}))\n",
      "        \n",
      "        return parameters"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 91
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "parameters = model(X_train , Y_train, X_test, Y_test)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Cost after epoch 0: 1.880317\n",
        "Cost after epoch 100: 0.915240"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 200: 0.522724"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 300: 0.293652"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 400: 0.173677"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 500: 0.094603"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 600: 0.057401"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 700: 0.029311"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 800: 0.016100"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 900: 0.007676"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 1000: 0.003377"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 1100: 0.001795"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 1200: 0.001447"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 1300: 0.001007"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Cost after epoch 1400: 0.000706"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEWCAYAAAB1xKBvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmcHFW5//HP07NPZklmSTLJTBZC\nQhIgJDAEUFYVCIiiV5AgKl7w5oKiv6t3Q/0pitffVdy4KopRMKhXRBEkILusGgKZAAkJISGE7Ntk\nmck2yWzP74+qgU4zSyeZnuqe+b5fr3pN16lTVc9JT/qZc6r6lLk7IiIiPYlFHYCIiGQGJQwREUmK\nEoaIiCRFCUNERJKihCEiIklRwhARkaQoYUi/ZWYPmdmVUcch0l8oYUivM7PVZva+qONw9wvc/Y6o\n4wAws6fM7NMRnLfMzO41s71mtsbMPtZNXTOz75jZ9nD5jplZ3PapZrbQzPaFP6cewr6zzWy5mbWb\n2adS1mBJKSUMyUhmlh11DB3SKZZO3AI0A8OAK4CfmdmxXdSdBXwIOAGYAnwA+GcAM8sF7gN+CwwB\n7gDuC8u73Te0CPgM8GJvNUwi4O5atPTqAqwG3tfFtouAl4EGYB4wJW7b9cAbwG7gVeDDcds+Bfwd\n+CGwHfivsOxvwPeAncCbwAVx+zwFfDpu/+7qjgWeCc/9OMEH7W+7aMPZwHrgP4HNwG8IPkQfAOrD\n4z8AVIf1vwW0AfuBPcBPwvKJwGPADmA58NFefh8GESSLCXFlvwG+3UX9ecCsuPWrgfnh6/OADYDF\nbV8LzOhp34Rz/A34VNS/o1oOb1EPQ/qMmU0Dbif4y7Mc+Dkw18zywipvAGcApcA3gN+aWVXcIU4B\nVhH8tfytuLLlQAVwE3Bb/FBIgu7q/g54IYzr68AnemjOcKAMGE3w13UM+FW4PgpoAn4C4O5fAZ4F\nrnP3Ine/zswGESSL3wFDgZnAT81scmcnM7OfmllDF8viLmKcALS6+4q4skVAVz2MY8PtndU9Fljs\n4ad+aHHC9q72lX5CCUP60izg5+7+vLu3eXB94QBwKoC7/9HdN7p7u7vfBbwOTI/bf6O7/9jdW929\nKSxb4+6/cPc2gmGSKoKE0plO65rZKOBk4Gvu3uzufwPm9tCWduAGdz/g7k3uvt3d/+Tu+9x9N0FC\nO6ub/S8CVrv7r8L2vAT8Cbi0s8ru/hl3H9zFMqWLcxQBuxLKGoHibuo3JtQtCpNq4rbEY3W3r/QT\n6Tz2Kv3PaOBKM/tcXFkuMALAzD4JfBEYE24rIugNdFjXyTE3d7xw933h51NRF+fvqm4FsMPd9yWc\nq6abttS7+/6OFTMrJBgum0EwPAVQbGZZYYJKNBo4xcwa4sqyCYaMesseoCShrIRg2C2Z+iXAHnd3\nM+vpWF3ueziBS3pSD0P60jrgWwl/HRe6+51mNhr4BXAdUO7ug4ElQPxfqKn68NkElIUf+h26Sxad\nxfKvwDHAKe5eApwZllsX9dcBTyf8WxS5+7WdnczMbjWzPV0sS7uIcQWQbWbj48pOALqqvzTc3lnd\npcCUhB7DlITtXe0r/YQShqRKjpnlxy3ZBAnhGjM7JbwNc5CZvd/Migku0DrBRWPM7B+B4/oiUHdf\nA9QBXzezXDM7jeAun0NRTHDdosHMyoAbErZvAY6KW38AmGBmnzCznHA52cwmdRHjNWFC6Wzp9FqB\nu+8F7gFuDP+t3w1cTNe9mF8DXzSzkWY2giAJzgm3PUVw4f7zZpZnZteF5U8ksS/hv2s+QQLt+N3Q\n50+G0RsmqfIgwQdox/J1d68D/ongYvBOYCXB3Uu4+6vA94HnCD5cjye4K6qvXAGcxtt3YN1FcH0l\nWTcDBcA2YD7wcML2/wEuMbOdZvaj8DrHeQQXuzcSDJd9B8ijd30mjGsrcCdwrbsvBTCzM8Khpg4/\nB+4HXiHo3f0lLMPdmwlum/0kwR1uVwEfCsu73Tf0KMHvwbuA2eHrM5GMYhpiFHknM7sLeM3dE3sK\nIgOWehgiQDgcNM7MYmY2g2Do5s9RxyWSTnSXlEhgOMF4fznBl/KuDW91FZGQhqRERCQpGpISEZGk\n9KshqYqKCh8zZkzUYYiIZIyFCxduc/fKZOr2q4QxZswY6urqog5DRCRjmNmaZOtqSEpERJKihCEi\nIklRwhARkaQoYYiISFKUMEREJClKGCIikhQlDBERScqATxjt7c5Pnnidp1fURx2KiEhaG/AJIxYz\nZj+ziieWbYk6FBGRtDbgEwbA8NJ8Nu/a33NFEZEBTAkDGF5awOZGJQwRke4oYQBVJflsUsIQEemW\nEgYwrDSf+j0HaGlrjzoUEZG0pYQBVJXm4w71uw9EHYqISNpSwiC46A1oWEpEpBtKGMDwkiBh6MK3\niEjXlDAIhqQA3VorItINJQygtCCH/JwYGxuaog5FRCRtKWEAZsaoskLW7tgXdSgiImkrZc/0NrPb\ngYuAre5+XCfb/x24Ii6OSUClu+8ws9XAbqANaHX32lTF2WF0+SDWbN+b6tOIiGSsVPYw5gAzutro\n7t9196nuPhX4EvC0u++Iq3JOuD3lyQJgTHkha7bvo73d++J0IiIZJ2UJw92fAXb0WDFwOXBnqmJJ\nxpiKQRxobdeFbxGRLkR+DcPMCgl6In+KK3bgUTNbaGazeth/lpnVmVldff3hT1E+pnwQAKs1LCUi\n0qnIEwbwAeDvCcNRp7v7icAFwGfN7Myudnb32e5e6+61lZWVhx3E6PJCANZs14VvEZHOpEPCmEnC\ncJS7bwh/bgXuBaanOoiq0gJys2Ks3qYehohIZyJNGGZWCpwF3BdXNsjMijteA+cBS1IdS1bMOKpy\nEMu37E71qUREMlIqb6u9EzgbqDCz9cANQA6Au98aVvsw8Ki7x/9ZPwy418w64vuduz+cqjjjTa4q\n4W8rt/XFqUREMk7KEoa7X55EnTkEt9/Gl60CTkhNVN2bPKKEe17awPY9BygvyosiBBGRtJUO1zDS\nxqSqEgCWbdKwlIhIIiWMOB0J49VNjRFHIiKSfpQw4pQNyqWiKI83tupOKRGRREoYCSqKctm+V0/e\nExFJpISRoKIoj+17m6MOQ0Qk7ShhJCgblMsOJQwRkXdQwkhQNiiXHXuUMEREEilhJKgoymX3gVYO\ntLZFHYqISFpRwkhQNij4wp6GpUREDqaEkaBsUC4A2zUsJSJyECWMBBVFYcJQD0NE5CBKGAk6ehg7\n9F0MEZGDKGEkKA+vYWhISkTkYEoYCUoKssmOmS56i4gkUMJIYGaUF+WybY+GpERE4ilhdGJ4aQGb\nGvdHHYaISFpRwujEiNJ8NjY0RR2GiEhaSVnCMLPbzWyrmXX6PG4zO9vMGs3s5XD5Wty2GWa23MxW\nmtn1qYqxKyMGF7CxYT/u3tenFhFJW6nsYcwBZvRQ51l3nxouNwKYWRZwC3ABMBm43MwmpzDOdxgx\nuICmljYa9rX05WlFRNJayhKGuz8D7DiMXacDK919lbs3A78HLu7V4HowcnA+ABs0LCUi8paor2Gc\nZmaLzOwhMzs2LBsJrIursz4s65SZzTKzOjOrq6+v75WgRgwuANB1DBGROFEmjBeB0e5+AvBj4M+H\ncxB3n+3ute5eW1lZ2SuBKWGIiLxTZAnD3Xe5+57w9YNAjplVABuAmriq1WFZnykflEtudoyNurVW\nROQtkSUMMxtuZha+nh7Gsh1YAIw3s7FmlgvMBOb2cWzUDClgVf2evjytiEhay07Vgc3sTuBsoMLM\n1gM3ADkA7n4rcAlwrZm1Ak3ATA/uY201s+uAR4As4HZ3X5qqOLtyQs1gnllRj7sT5jURkQEtZQnD\n3S/vYftPgJ90se1B4MFUxJWsaaOGcM+LG1i/s4massIoQxERSQtR3yWVtqbVDAbgxbU7I45ERCQ9\nKGF0YeLwYvJzYry0tiHqUERE0oISRheys2JMriph2aZdUYciIpIWlDC6cczwEpZv2a05pUREUMLo\n1jHDimjY10L9bj0bQ0RECaMbE4YXA7B8y+6IIxERiZ4SRjeOGRYmjM1KGCIiShjdKC/Ko6IoVwlD\nRAQljB5NHF7Css26U0pERAmjB8eOLGH55t00t7ZHHYqISKSUMHpw3IhSWtqc17dqWEpEBjYljB4c\nO6IEgKUbNCwlIgObEkYPxpQPYlBuFks2NkYdiohIpJQwehCLGcdXl/LyOs0pJSIDmxJGEqaPKWPJ\nhkb2HGiNOhQRkcgoYSTh5LFltDssXKOpzkVk4FLCSMKJo4aQFTMWvLkj6lBERCKTsoRhZreb2VYz\nW9LF9ivMbLGZvWJm88zshLhtq8Pyl82sLlUxJmtQXjaTq0r0MCURGdBS2cOYA8zoZvubwFnufjzw\nTWB2wvZz3H2qu9emKL5DMqmqmBWahFBEBrCUJQx3fwbocgzH3ee5e8ef7POB6lTF0hsmDCtm255m\ntu3RVOciMjClyzWMq4GH4tYdeNTMFprZrIhiOsgx4VTn6mWIyEAVecIws3MIEsZ/xhWf7u4nAhcA\nnzWzM7vZf5aZ1ZlZXX19fcri7JjqfIVmrhWRASrShGFmU4BfAhe7+/aOcnffEP7cCtwLTO/qGO4+\n291r3b22srIyZbFWFucxuDCH5Vv2pOwcIiLpLLKEYWajgHuAT7j7irjyQWZW3PEaOA/o9E6rvmRm\nTBpewuL1+sa3iAxM2ak6sJndCZwNVJjZeuAGIAfA3W8FvgaUAz81M4DW8I6oYcC9YVk28Dt3fzhV\ncR6KU48q5+a/rqBhXzODC3OjDkdEpE+lLGG4++U9bP808OlOylcBJ7xzj+idNq6cHz4Oz7+5g/OP\nHR51OCIifSryi96Z5ISaUvJzYjz3xvaeK4uI9DNKGIcgLzuLk8eUKWGIyICkhHGITj2qnOVbdrNd\nX+ATkQFGCeMQnTauHID5qzQRoYgMLEoYh2jKyFKK8rKZ98a2qEMREelTShiHKDsrxsljhvDcKl3H\nEJGBRQnjMJw2rpxV9XvZsmt/1KGIiPQZJYzD8K5xFQC6W0pEBhQljMMwqaqEkvxsJQwRGVCUMA5D\nVsw45ahyXccQkQFFCeMwvWtcOWt37GP9zn1RhyIi0ieUMA5Tx/cxNCwlIgOFEsZhmjC0mLJBudz7\n0gbWblcvQ0T6PyWMwxSLGR+YUsW8N7Yz6zd1UYcjIpJyKZvefCD4xsXHUZiXzexnVrG/pY38nKyo\nQxIRSRn1MI7QcSNKaWt3Vm7Vo1tFpH9TwjhCk6qKAXh1066IIxERSS0ljCM0unwQBTlZvLZpd9Sh\niIikVEoThpndbmZbzWxJF9vNzH5kZivNbLGZnRi37Uozez1crkxlnEciK2YcM7yYZephiEg/l1TC\nMLNLkynrxBxgRjfbLwDGh8ss4GfhscuAG4BTgOnADWY2JJlYo3D8yFIWr2+gta096lBERFIm2R7G\nl5IsO4i7PwN096Shi4Ffe2A+MNjMqoDzgcfcfYe77wQeo/vEE6mTx5axt7mNZRqWEpF+rNvbas3s\nAuBCYKSZ/ShuUwnQ2gvnHwmsi1tfH5Z1Vd5ZjLMIeieMGjWqF0I6dCePCTo/L6zewfHVpZHEICKS\naj31MDYCdcB+YGHcMpegFxA5d5/t7rXuXltZWRlJDFWlBVQPKWDBm3psq4j0X932MNx9EbDIzH7n\n7i0A4bWEmnCo6EhtAGri1qvDsg3A2QnlT/XC+VLm3eMquH/xRhr2NTO4MDfqcEREel2y1zAeM7OS\n8GL0i8AvzOyHvXD+ucAnw7ulTgUa3X0T8AhwnpkNCRPUeWFZ2vrH08ewr7mNO+atiToUEZGUSDZh\nlLr7LuAfCC5SnwK8t6edzOxO4DngGDNbb2ZXm9k1ZnZNWOVBYBWwEvgF8BkAd98BfBNYEC43hmVp\na+LwEt43aSh3PLea5lbdLSUi/U+yc0llh3cvfRT4SrIHd/fLe9juwGe72HY7cHuy50oHl08fxePL\ntvL0inrOnTws6nBERHpVsj2MGwmGhN5w9wVmdhTweurCykxnTqikoiiXuxeu67myiEiGSSphuPsf\n3X2Ku18brq9y94+kNrTMk5MV49LaGh5ZuoX5enyriPQzyX7Tu9rM7g2n+dhqZn8ys+pUB5eJPvee\noxlVVsgN9y2NOhQRkV6V7JDUrwjuaBoRLveHZZKgMDeby06uYfmW3TQ2tUQdjohIr0k2YVS6+6/c\nvTVc5gDRfEsuA0weUQLAa5qQUET6kWQTxnYz+7iZZYXLxwEN0ndhclWQMPSMDBHpT5JNGFcR3FK7\nGdgEXAJ8KkUxZbyhxXmUD8rVlOci0q8k+z2MG4ErO6YDCb/x/T2CRCIJzIxJVSXqYYhIv5JsD2NK\n/NxR4beup6UmpP5h+tgylmzYxcI1af0FdRGRpCWbMGLxDzAKexjJ9k4GpKtPH0tVaT5f/fNSgi+0\ni4hktmQTxveB58zsm2b2TWAecFPqwsp8g/Ky+fx7x/Pqpl0sXt8YdTgiIkcs2W96/5pg4sEt4fIP\n7v6bVAbWH1x4fBW52THufWlD1KGIiByxpIeV3P1V4NUUxtLvlBbk8L5JQ7nnxfVcclI1x43U0/hE\nJHMlOyQlh+nfz59IcX4On7jtefa3tEUdjojIYVPCSLGxFYP47qVT2LmvhUeWbo46HBGRw6aE0QdO\nHVvOyMEF3L1wfdShiIgcNiWMPhCLGTNPruHZ17dx4/26DCQimSmlCcPMZpjZcjNbaWbXd7L9h2b2\ncrisMLOGuG1tcdvmpjLOvvCZc47mo7XV3P73N9mya3/U4YiIHLKUJQwzywJuAS4AJgOXm9nk+Dru\n/gV3n+ruU4EfA/fEbW7q2ObuH0xVnH0lK2ZcccpoAOpW7+yhtohI+kllD2M6sDJ8Ol8z8Hvg4m7q\nXw7cmcJ4Ijd5RAkFOVksWK3pQkQk86QyYYwE4h9uvT4sewczGw2MBZ6IK843szozm29mH+rqJGY2\nK6xXV19f3xtxp0xOVoypNYOp0/xSIpKB0uWi90zgbneP/6LCaHevBT4G3Gxm4zrb0d1nu3utu9dW\nVqb/M51OHlvGqxt3samxKepQREQOSSoTxgagJm69OizrzEwShqPcfUP4cxXwFP1kdtxLT6rGzPjl\ns29GHYqIyCFJZcJYAIw3s7FmlkuQFN5xt5OZTQSGAM/FlQ0xs7zwdQXwbvrJtCQ1ZYVcfMIIfvf8\nWlZs2R11OCIiSUtZwnD3VuA64BFgGfAHd19qZjeaWfxdTzOB3/vBc4BPAurMbBHwJPDtcC6rfuHf\nZxxDcX42V81ZoOlCRCRjWH96VkNtba3X1dVFHUZSnn29nk/c9gI3XTKFj9bW9LyDiEgKmNnC8Hpx\nj9LloveAc/rRFRwzrJg75q3WA5ZEJCMoYUTEzLj69LEs3biLHz+xMupwRER6pIQRoUtrq/mHaSP5\nwWMrWL1tb9ThiIh0SwkjQmbGv55/DAAPa+pzEUlzShgRGzm4gCnVpTy8RAlDRNKbEkYaOP/Y4by8\nroElGxqjDkVEpEtKGGngilNGMbQ4j3/9wyIOtOp7GSKSnpQw0sDgwly+/ZHjWb5lN7f9TVOGiEh6\nUsJIE++ZOIzzJg/jx39dycYGTUwoIulHCSONfPWiybS7862/LIs6FBGRd1DCSCM1ZYV89pyj+csr\nm3jwlU1RhyMichAljDRzzVnjmDZqMP9x92LeqN8TdTgiIm9RwkgzudkxbvnYieRmx7j2twvZc6A1\n6pBERAAljLQ0YnABP5o5jTfq92oKdBFJG0oYaer08RX84KMn8MKbO5gzb3XU4YiIKGGks4unjuSc\nYyr56ZMradjXHHU4IjLAKWGkuesvmMSeA63c8qSmQBeRaKU0YZjZDDNbbmYrzez6TrZ/yszqzezl\ncPl03LYrzez1cLkylXGms2OGF/ORE6u5Y94a1u/cF3U4IjKApSxhmFkWcAtwATAZuNzMJndS9S53\nnxouvwz3LQNuAE4BpgM3mNmQVMWa7r5w7gQAbnnyjYgjEZGBLJU9jOnASndf5e7NwO+Bi5Pc93zg\nMXff4e47gceAGSmKM+2NGFzAzOk1/LFuHU8u3xp1OCIyQKUyYYwE1sWtrw/LEn3EzBab2d1mVnOI\n+2Jms8yszszq6uvreyPutHTdOUdTU1bIP/5qgb4FLiKRiPqi9/3AGHefQtCLuONQD+Dus9291t1r\nKysrez3AdDG0JJ+H/+UMJg4v5jsPv6Yv9IlIn0tlwtgA1MStV4dlb3H37e5+IFz9JXBSsvsORHnZ\nWfznjIms2b6PaTc+ytfuW6Iv9YlIn0llwlgAjDezsWaWC8wE5sZXMLOquNUPAh3TtD4CnGdmQ8KL\n3eeFZQPeOROHcvc1p3HJSdX8+rk1/PLZVVGHJCIDRHaqDuzurWZ2HcEHfRZwu7svNbMbgTp3nwt8\n3sw+CLQCO4BPhfvuMLNvEiQdgBvdfUeqYs00tWPKqB1Txtod+/jd82u59uyjyYpZ1GGJSD9n7h51\nDL2mtrbW6+rqog6jzzy8ZBPX/PZFfnrFiVx4fFXPO4iIJDCzhe5em0zdqC96yxF436RhHD20iO8+\nspzm1vaowxGRfk4JI4NlZ8X48oUTeXPbXr736PKowxGRfk4JI8Odc8xQPnHqaGY/s4o5f38z6nBE\npB9L2UVv6Rtmxtc/eCybd+3nGw+8yvqdTfzLuRMoytNbKyK9Sz2MfiArZvxo5jQ+PHUkt//9TX74\n2IqoQxKRfkgJo58oyM3iB5dN5YLjq7h74Xp9oU9Eep0SRj9zxfRRNDa18Nv5a6IORUT6GSWMfua0\nceWcc0wl33pwGXMXbYw6HBHpR3RltJ8xM356xUlc+asX+OJdL5OfHWPbnmbefXQ5o8sHRR2eiGQw\n9TD6oYLcLG67spZjR5Qw6zcL+fK9r/DNB16NOiwRyXBKGP1UcX4Od1w1nfdOHArAgtU7aWvvP9PA\niEjfU8LoxwYX5nLbp07mJx+bRmNTCzc98hqbGpuiDktEMpQSxgBwxvjgwVI/f3oVH//l87y2eZdu\nuxWRQ6aEMQCUFgTDU9+8+FhWb9/HjJuf5at/XhJ1WCKSYXSX1ABx1oRKoJIp1YP58RMruW/RRk49\nqpyjhxZxQs3gqMMTkQygHsYAc0LNYL547gSaW9v51z8u4iM/m8d9Lw/4p9+KSBLUwxiAJo8o4eOn\njqK0IIf5q3Zww9ylnDNxKCX5OVGHJiJpTD2MAeq/PnQ8/37+RL7xwWNp2NfCtx5YRkubHsIkIl1L\nacIwsxlmttzMVprZ9Z1s/6KZvWpmi83sr2Y2Om5bm5m9HC5zUxnnQHbcyFL+6Yyx3FW3jvf/6Fnm\nvbEt6pBEJE2lLGGYWRZwC3ABMBm43MwmJ1R7Cah19ynA3cBNcdua3H1quHwwVXEKfOX9k/n5J06i\nubWdT972Ave8uD7qkEQkDaWyhzEdWOnuq9y9Gfg9cHF8BXd/0t33havzgeoUxiPdOP/Y4dz/udM5\neUwZX/zDIr4+d6meEy4iB0llwhgJrItbXx+WdeVq4KG49XwzqzOz+Wb2oa52MrNZYb26+vr6I4t4\ngCvOz+HXV0/n6tPHMmfeaj5x2/Ms37ybJ5dvZVNjEwda29i9vyXqMEUkImlxl5SZfRyoBc6KKx7t\n7hvM7CjgCTN7xd3fSNzX3WcDswFqa2s1WdIRysmK8dWLJnPcyBL+7Y+LOf/mZwAYXJjDuMoidu9v\n4ZF/ORMzizhSEelrqUwYG4CauPXqsOwgZvY+4CvAWe5+oKPc3TeEP1eZ2VPANOAdCUNS48PTqqkZ\nUshrm3czrCSfz/7uRRau2QnAyq17GD+sOOIIRaSvpXJIagEw3szGmlkuMBM46G4nM5sG/Bz4oLtv\njSsfYmZ54esK4N2A5ufuY7Vjyvj4qaM5d/Iwbr5sKteePQ6Ax5dt7WFPEemPUtbDcPdWM7sOeATI\nAm5396VmdiNQ5+5zge8CRcAfwyGOteEdUZOAn5tZO0FS+7a7K2FE6MLjq7jw+Cr+9vo27nxhLVOq\nS3n30RVRhyUifcjc+8+wf21trdfV1UUdRr/25PKtfOlPr7B1935mnTmOySNK+MCUKl3TEMlQZrbQ\n3WuTqZsWF70lc5xzzFCe+LezuGrOAm59Orik9NLanfzTGUcxtDiP7CxNHiDSX6mHIYfF3dmxt5kf\nPr6C385fC8DQ4jzeM3EoL67dyV2zTmPIoNyIoxSRnhxKD0MJQ47Yii27mb9qO7c+9QYbG/cD8JET\nq/nepVMOGqpydw1diaQZDUlJn5owrJgJw4o5/9jhLN3YSN3qnfz0qTdo2NfMDy6bSk6W8c0HXuXR\npVt44POnU1VaEHXIInIYlDCk1wwryWdYST5nTxhKRVEe//3QMi768bNkx2Ks3r4Xd/j9C+v4wrkT\nog5VRA6DrlBKr4vFjKtOH8uvrzqFEaUF5Odk8b9Xn8JZEyr52dNvMPGrD/G5O1+ibvUO+tOQqEh/\npx6GpMxp48o5bdxpB5Ut37yb6WPLeHL5Vu5ftJHjRpZw82XTOHpoUURRikiydNFbIrGvuZU/v7SR\nHzy2nAMt7YwfVsRZE4Zy9RljKcrT3zEifUV3SUnGWL1tLzc/voK1O/bx0roGaoYUcsb4CkaVFfLR\n2hrdmiuSYkoYkpGeX7Wd/37oNdZs38vOfS0MLc7j5DFlVBTlcsWpo5mgCQ9Fep0ShmS8JRsa+c7D\nr7GhoYlNDftpa3fOPXYY02oGU5Kfw7ihg3h82VbOnTyME0cNiTpckYylhCH9yrY9B7j58RU8unQL\nW3cfOGhbXnaMmy6ZwjHDi8mOxageEtyVJSLJUcKQfmv7ngPs2t/KonUNjC4v5L/+suyt53R0uPD4\n4Vx71tGMH1ZEXnaMvc1tDMrN0rfMRTqhhCEDRktbO3cvXE9OVoysGCzbtJs581bT3NqOGRjQ7jCl\nupSaskKqhxTw6dOP4sW1Oxleks8JNYOjboJIpJQwZEDb1NjES2sbWLFlN23tTnYsxm1/W0VLm9PU\n0nZQ3QnDijh2RCnXnDWOVzc1YhgfOGEEWbGueyPt7U5jU4vu4JJ+QQlDJEHjvhays4wnl29l7Y59\nnDymjL+v3Mbi9Y387fVtNLdf5YM2AAANC0lEQVS1v1W3sjiP4SX5HD20iGmjBrN9TzMVxXlMripm\n4vASvvrnJTzwyib+8M+nsauphQdf2cSX3z+JkvycCFsocniUMEQOwar6PSxa30D1kEI2Ne7nqeVb\n2bG3mRfX7GTX/taD6pqBO+TnxDCMA61tbw15XTSlirZ2eOHN7Zw5oZLR5YVMriolZkESAtjQ0MTI\nwQXhsXRNRaKXNgnDzGYA/0PwiNZfuvu3E7bnAb8GTgK2A5e5++pw25eAq4E24PPu/khP51PCkN7U\n1NzGjn3NDC/JZ8uu/SzduIulGxsBuGhKFb/6+2qK8rIZP6yYmx5+7a07uIaV5LFl18F3cw0rySMn\nK8b6nU0cPbSIdTv2MbZiENNGDaEgJ4ujKgex90Arbe5UFOUxorSAmMF3H13O+yYN46p3jyUWg+xY\njJgdXrJpam6jubWd0kL1hORtaZEwzCwLWAGcC6wHFgCXxz+b28w+A0xx92vMbCbwYXe/zMwmA3cC\n04ERwOPABHdvSzxPPCUMiVLDvmZ2NbVSU1YQfH+kcT/LNu2itc1ZvL6BPQfamFxVzFMr6plcVcKG\nhiYWrWvo9NpKh/ycGPtb2g8qMwseVlUcDoHF/x+uLM6joiiPDQ1NFOZmMX5oMc1t7RTmZPHQks3s\n3t/CR06qpqm5jfKiXCqK8hiUm82BtnZiBoMLcinKz6a93Wlrd1rbnXYPf7Y7udkxjh5aRLs7+1va\n2d/Sxv6WNtxh/LAiCnOz30poMYOYGTGz4AaEuPXDTXrS+9IlYZwGfN3dzw/XvwTg7v8dV+eRsM5z\nZpYNbAYqgevj68bX6+6cShiSidyd9TubKCnIITcrRv3uA2xsbKJ+9wHeNa6c51ZtZ/3OJtrCD/GW\ntnY2Ne6nqTkuyRjgwZDXrqYWKorz2Lm3mc2N+8nLyWLX/hZGlOZTWpjLqxsbKS3IZee+Ztraox2S\nzoq9nTyywsTSIT6ddCSXg1JMT3V72B5fbl0cuKP84P3ja/ZUt/Ok+FbdwzxWYrhlhbncfe27Oj1X\nT9LlAUojgXVx6+uBU7qq4+6tZtYIlIfl8xP2HdnZScxsFjALYNSoUb0SuEhfMjNqygrfWh9VXsio\n8rfXL5oy4ojP0drWTlbMMDNa29rJzoq9dbfXngOt5GXHaHdoaGpm74FWYmZkxeKWcH3vgTZWbdtD\nTlaM/JwY+TlZ5Odk0dbuvL5lDy1t7bS70+5BIux43e6Ov1UGbe3+1ut2d9o86MF0iP871jst67zu\n22Vx2w8qf+cxOjvXweVdxHVYx4qLvcu63sP+76xb3Ec3XGT8tKDuPhuYDUEPI+JwRNJSdlbsHa9j\nMWPIoNyDbg8eXprf47GOry7ttPzkMWVHGKWku1Q+QGkDUBO3Xh2WdVonHJIqJbj4ncy+IiLSh1KZ\nMBYA481srJnlAjOBuQl15gJXhq8vAZ7woI81F5hpZnlmNhYYD7yQwlhFRKQHKRuSCq9JXAc8QnBb\n7e3uvtTMbgTq3H0ucBvwGzNbCewgSCqE9f4AvAq0Ap/t6Q4pERFJLX1xT0RkADuUu6RSOSQlIiL9\niBKGiIgkRQlDRESSooQhIiJJ6VcXvc2sHlhzmLtXANt6MZwoqS3pp7+0A9SWdHW4bRnt7pXJVOxX\nCeNImFldsncKpDu1Jf30l3aA2pKu+qItGpISEZGkKGGIiEhSlDDeNjvqAHqR2pJ++ks7QG1JVylv\ni65hiIhIUtTDEBGRpChhiIhIUgZ8wjCzGWa23MxWmtn1UcdzqMxstZm9YmYvm1ldWFZmZo+Z2evh\nzyFRx9kZM7vdzLaa2ZK4sk5jt8CPwvdpsZmdGF3k79RFW75uZhvC9+ZlM7swbtuXwrYsN7Pzo4m6\nc2ZWY2ZPmtmrZrbUzP5PWJ5x7003bcm498bM8s3sBTNbFLblG2H5WDN7Poz5rvBxEoSPh7grLH/e\nzMYccRDuPmAXgmnX3wCOAnKBRcDkqOM6xDasBioSym4Crg9fXw98J+o4u4j9TOBEYElPsQMXAg8R\nPM74VOD5qONPoi1fB/6tk7qTw9+1PGBs+DuYFXUb4uKrAk4MXxcDK8KYM+696aYtGffehP++ReHr\nHOD58N/7D8DMsPxW4Nrw9WeAW8PXM4G7jjSGgd7DmA6sdPdV7t4M/B64OOKYesPFwB3h6zuAD0UY\nS5fc/RmC56DE6yr2i4Ffe2A+MNjMqvom0p510ZauXAz83t0PuPubwEqC38W04O6b3P3F8PVuYBkw\nkgx8b7ppS1fS9r0J/333hKs54eLAe4C7w/LE96Xj/bobeK+Z2ZHEMNATxkhgXdz6err/ZUpHDjxq\nZgvNbFZYNszdN4WvNwPDogntsHQVe6a+V9eFwzS3xw0NZkxbwmGMaQR/zWb0e5PQFsjA98bMsszs\nZWAr8BhBD6jB3VvDKvHxvtWWcHsjUH4k5x/oCaM/ON3dTwQuAD5rZmfGb/SgP5qR905ncuyhnwHj\ngKnAJuD70YZzaMysCPgT8C/uvit+W6a9N520JSPfG3dvc/epQDVBz2diX55/oCeMDUBN3Hp1WJYx\n3H1D+HMrcC/BL9GWjiGB8OfW6CI8ZF3FnnHvlbtvCf+DtwO/4O2hjbRvi5nlEHzA/q+73xMWZ+R7\n01lbMvm9AXD3BuBJ4DSCIcCOx23Hx/tWW8LtpcD2IznvQE8YC4Dx4V0GuQQXhuZGHFPSzGyQmRV3\nvAbOA5YQtOHKsNqVwH3RRHhYuop9LvDJ8I6cU4HGuOGRtJQwjv9hgvcGgrbMDO9iGQuMB17o6/i6\nEo5z3wYsc/cfxG3KuPemq7Zk4ntjZpVmNjh8XQCcS3BN5kngkrBa4vvS8X5dAjwR9gwPX9RX/qNe\nCO7wWEEwFviVqOM5xNiPIrijYxGwtCN+gnHKvwKvA48DZVHH2kX8dxIMB7QQjL1e3VXsBHeI3BK+\nT68AtVHHn0RbfhPGujj8z1sVV/8rYVuWAxdEHX9CW04nGG5aDLwcLhdm4nvTTVsy7r0BpgAvhTEv\nAb4Wlh9FkNRWAn8E8sLy/HB9Zbj9qCONQVODiIhIUgb6kJSIiCRJCUNERJKihCEiIklRwhARkaQo\nYYiISFKUMCTtmdm88OcYM/tYLx/7y52dK1XM7ENm9rUUHfvLPdc65GMeb2Zzevu4kpl0W61kDDM7\nm2CG0YsOYZ9sf3uenc6273H3ot6IL8l45gEfdPdtR3icd7QrVW0xs8eBq9x9bW8fWzKLehiS9sys\nY4bObwNnhM8v+EI4Edt3zWxBOIncP4f1zzazZ81sLvBqWPbncILGpR2TNJrZt4GC8Hj/G3+u8FvL\n3zWzJRY8b+SyuGM/ZWZ3m9lrZva/HTOAmtm3LXjuwmIz+14n7ZgAHOhIFmY2x8xuNbM6M1thZheF\n5Um3K+7YnbXl4xY8P+FlM/u5mWV1tNHMvmXBcxXmm9mwsPzSsL2LzOyZuMPfTzALggx0UX97UYuW\nnhZgT/jzbOCBuPJZwP8NX+cBdQTPMDgb2AuMjavb8a3kAoJvyZbHH7uTc32EYDbQLIJZWdcSPFvh\nbIJZP6sJ/uB6juDbxOUE3wzu6LUP7qQd/wh8P259DvBweJzxBN8Qzz+UdnUWe/h6EsEHfU64/lPg\nk+FrBz4Qvr4p7lyvACMT4wfeDdwf9e+BluiXjgmrRDLRecAUM+uYR6eU4IO3GXjBg+cZdPi8mX04\nfF0T1utuIrbTgTvdvY1g0r2ngZOBXeGx1wNYMNX0GGA+sB+4zcweAB7o5JhVQH1C2R88mADvdTNb\nRTD76KG0qyvvBU4CFoQdoALeniywOS6+hQRzEgH8HZhjZn8A7nn7UGwFRiRxTunnlDAkkxnwOXd/\n5KDC4FrH3oT19wGnufs+M3uK4C/5w3Ug7nUbkO3urWY2neCD+hLgOoIH28RrIvjwj5d4EdFJsl09\nMOAOd/9SJ9ta3L3jvG2EnwPufo2ZnQK8H1hoZie5+3aCf6umJM8r/ZiuYUgm2U3wmM0OjwDXWjB9\nNWY2IZy1N1EpsDNMFhMJHmvZoaVj/wTPApeF1xMqCR7B2uWspRY8b6HU3R8EvgCc0Em1ZcDRCWWX\nmlnMzMYRTCK3/BDalSi+LX8FLjGzoeExysxsdHc7m9k4d3/e3b9G0BPqmOZ7Am/P5ioDmHoYkkkW\nA21mtohg/P9/CIaDXgwvPNfT+eNoHwauMbNlBB/I8+O2zQYWm9mL7n5FXPm9BM8aWETwV/9/uPvm\nMOF0phi4z8zyCf66/2IndZ4Bvm9mFvcX/lqCRFQCXOPu+83sl0m2K9FBbTGz/0vwNMYYwSy6nwXW\ndLP/d81sfBj/X8O2A5wD/CWJ80s/p9tqRfqQmf0PwQXkx8PvNzzg7nf3sFtkzCwPeJrgyY5d3p4s\nA4OGpET61v8DCqMO4hCMAq5XshBQD0NERJKkHoaIiCRFCUNERJKihCEiIklRwhARkaQoYYiISFL+\nP/xF9isRjo4BAAAAAElFTkSuQmCC\n",
       "text": [
        "<matplotlib.figure.Figure at 0x7f280811b400>"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Parameters have benn trained!\n",
        "Train Accuracy:"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1.0\n",
        "Test Accuracy: 0.875\n"
       ]
      }
     ],
     "prompt_number": 92
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}